{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n",
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.decomposition import truncated_svd\n",
    "import category_encoders as ce\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn import preprocessing\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    yield\n",
    "    print('[{' + name + '}] done in {' + str(round(time.time() - t0, 3)) + '} s')\n",
    "\n",
    "num_rows = None\n",
    "EPS = 1e-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_ann_suresh_features2_neptune_1.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_ann_suresh_features2_neptune_1.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('/media/limbo/Home-Credit/data/application_test.csv', nrows=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip', nrows= num_rows)\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/selected_features2.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME_CONTRACT_TYPE_SURESH</th>\n",
       "      <th>CODE_GENDER_SURESH</th>\n",
       "      <th>FLAG_OWN_CAR_SURESH</th>\n",
       "      <th>AMT_INCOME_TOTAL_SURESH</th>\n",
       "      <th>AMT_ANNUITY_SURESH</th>\n",
       "      <th>AMT_GOODS_PRICE_SURESH</th>\n",
       "      <th>NAME_INCOME_TYPE_SURESH</th>\n",
       "      <th>NAME_EDUCATION_TYPE_SURESH</th>\n",
       "      <th>NAME_HOUSING_TYPE_SURESH</th>\n",
       "      <th>REGION_POPULATION_RELATIVE_SURESH</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOC_SUM_SURESH</th>\n",
       "      <th>DEF_30_60_SOCIAL_SUM_SURESH</th>\n",
       "      <th>OBS_30_60_SOCIAL_SUM_SURESH</th>\n",
       "      <th>TOTAL_CREDIT_CHECKS_YEAR_SURESH</th>\n",
       "      <th>PER_CREDIT_CHECKS_MON_SURESH</th>\n",
       "      <th>TOTAL_CREDIT_CHECKS_MON_SURESH</th>\n",
       "      <th>TOTAL_CREDIT_CHECKS_QRT_SURESH</th>\n",
       "      <th>PER_CREDIT_CHECKS_MON_3MON_SURESH</th>\n",
       "      <th>LIVE_IND_SUM_SURESH</th>\n",
       "      <th>NEW_INC_BY_ORG_SURESH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>768216</td>\n",
       "      <td>0.110012</td>\n",
       "      <td>84513</td>\n",
       "      <td>106619</td>\n",
       "      <td>0.792664</td>\n",
       "      <td>NaN</td>\n",
       "      <td>157500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>768216</td>\n",
       "      <td>0.110012</td>\n",
       "      <td>84513</td>\n",
       "      <td>106619</td>\n",
       "      <td>0.792664</td>\n",
       "      <td>38.244017</td>\n",
       "      <td>135000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>768216</td>\n",
       "      <td>0.110012</td>\n",
       "      <td>84513</td>\n",
       "      <td>106619</td>\n",
       "      <td>0.792664</td>\n",
       "      <td>NaN</td>\n",
       "      <td>135000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.008019</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>768216</td>\n",
       "      <td>0.110012</td>\n",
       "      <td>84513</td>\n",
       "      <td>106619</td>\n",
       "      <td>0.792664</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>157500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>768216</td>\n",
       "      <td>0.110012</td>\n",
       "      <td>84513</td>\n",
       "      <td>106619</td>\n",
       "      <td>0.792664</td>\n",
       "      <td>48.147723</td>\n",
       "      <td>157500.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 547 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   NAME_CONTRACT_TYPE_SURESH  CODE_GENDER_SURESH  FLAG_OWN_CAR_SURESH  \\\n",
       "0                          1                   2                    1   \n",
       "1                          1                   1                    1   \n",
       "2                          2                   2                    2   \n",
       "3                          1                   1                    1   \n",
       "4                          1                   2                    1   \n",
       "\n",
       "   AMT_INCOME_TOTAL_SURESH  AMT_ANNUITY_SURESH  AMT_GOODS_PRICE_SURESH  \\\n",
       "0                 202500.0             24700.5                351000.0   \n",
       "1                 270000.0             35698.5               1129500.0   \n",
       "2                  67500.0              6750.0                135000.0   \n",
       "3                 135000.0             29686.5                297000.0   \n",
       "4                 121500.0             21865.5                513000.0   \n",
       "\n",
       "   NAME_INCOME_TYPE_SURESH  NAME_EDUCATION_TYPE_SURESH  \\\n",
       "0                        5                           4   \n",
       "1                        4                           2   \n",
       "2                        5                           4   \n",
       "3                        5                           4   \n",
       "4                        5                           4   \n",
       "\n",
       "   NAME_HOUSING_TYPE_SURESH  REGION_POPULATION_RELATIVE_SURESH  \\\n",
       "0                         2                           0.018801   \n",
       "1                         2                           0.003541   \n",
       "2                         2                           0.010032   \n",
       "3                         2                           0.008019   \n",
       "4                         2                           0.028663   \n",
       "\n",
       "           ...            FLAG_DOC_SUM_SURESH  DEF_30_60_SOCIAL_SUM_SURESH  \\\n",
       "0          ...                              1                            4   \n",
       "1          ...                              1                            0   \n",
       "2          ...                              0                            0   \n",
       "3          ...                              1                            0   \n",
       "4          ...                              0                            0   \n",
       "\n",
       "   OBS_30_60_SOCIAL_SUM_SURESH  TOTAL_CREDIT_CHECKS_YEAR_SURESH  \\\n",
       "0                            4                           768216   \n",
       "1                            2                           768216   \n",
       "2                            0                           768216   \n",
       "3                            4                           768216   \n",
       "4                            0                           768216   \n",
       "\n",
       "   PER_CREDIT_CHECKS_MON_SURESH  TOTAL_CREDIT_CHECKS_MON_SURESH  \\\n",
       "0                      0.110012                           84513   \n",
       "1                      0.110012                           84513   \n",
       "2                      0.110012                           84513   \n",
       "3                      0.110012                           84513   \n",
       "4                      0.110012                           84513   \n",
       "\n",
       "   TOTAL_CREDIT_CHECKS_QRT_SURESH  PER_CREDIT_CHECKS_MON_3MON_SURESH  \\\n",
       "0                          106619                           0.792664   \n",
       "1                          106619                           0.792664   \n",
       "2                          106619                           0.792664   \n",
       "3                          106619                           0.792664   \n",
       "4                          106619                           0.792664   \n",
       "\n",
       "   LIVE_IND_SUM_SURESH  NEW_INC_BY_ORG_SURESH  \n",
       "0                  NaN               157500.0  \n",
       "1            38.244017               135000.0  \n",
       "2                  NaN               135000.0  \n",
       "3            51.000000               157500.0  \n",
       "4            48.147723               157500.0  \n",
       "\n",
       "[5 rows x 547 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 547)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "neptune = pd.read_csv('../data/neptue_features_0.csv', header=0, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>...</th>\n",
       "      <th>POPULAR_AMT_GOODS_PRICE_SURESH</th>\n",
       "      <th>INSTALLMENT_PAID_LATE_DAYS_LAST_LOAN_MEAN_SURESH</th>\n",
       "      <th>MEAN_PAYMENT_CC_SURESH</th>\n",
       "      <th>DAYS_EMPLOYED_DIFF_DAYS_BIRTH_SURESH</th>\n",
       "      <th>TREND_COEF_PAID_LATE_SURESH</th>\n",
       "      <th>AMT_INCOME_TOTAL_12_AMT_ANNUITY_SURESH</th>\n",
       "      <th>number_of_installments_paid_late_03</th>\n",
       "      <th>mean_number_of_days_paid_late_03</th>\n",
       "      <th>max_number_of_days_paid_late_03</th>\n",
       "      <th>mean_amont_paid_late_03</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-20.421053</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8824.0</td>\n",
       "      <td>0.157343</td>\n",
       "      <td>-7825.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.428571</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15577.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-13198.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-7.666667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18821.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1125.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15966.0</td>\n",
       "      <td>1.181081</td>\n",
       "      <td>-18436.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-2.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16894.0</td>\n",
       "      <td>-0.504714</td>\n",
       "      <td>-11740.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 362 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  AMT_ANNUITY  AMT_CREDIT  AMT_GOODS_PRICE  AMT_INCOME_TOTAL  \\\n",
       "0      0      24700.5    406597.5         351000.0          202500.0   \n",
       "1      1      35698.5   1293502.5        1129500.0          270000.0   \n",
       "2      2       6750.0    135000.0         135000.0           67500.0   \n",
       "3      3      29686.5    312682.5         297000.0          135000.0   \n",
       "4      4      21865.5    513000.0         513000.0          121500.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                        0.0                         0.0   \n",
       "1                        0.0                         0.0   \n",
       "2                        0.0                         0.0   \n",
       "3                        NaN                         NaN   \n",
       "4                        0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK           ...            \\\n",
       "0                         0.0           ...             \n",
       "1                         0.0           ...             \n",
       "2                         0.0           ...             \n",
       "3                         NaN           ...             \n",
       "4                         0.0           ...             \n",
       "\n",
       "   POPULAR_AMT_GOODS_PRICE_SURESH  \\\n",
       "0                               0   \n",
       "1                               0   \n",
       "2                               0   \n",
       "3                               0   \n",
       "4                               0   \n",
       "\n",
       "   INSTALLMENT_PAID_LATE_DAYS_LAST_LOAN_MEAN_SURESH  MEAN_PAYMENT_CC_SURESH  \\\n",
       "0                                        -20.421053                     NaN   \n",
       "1                                         -4.428571                     NaN   \n",
       "2                                         -7.666667                     NaN   \n",
       "3                                         -4.500000                     0.0   \n",
       "4                                         -2.250000                     NaN   \n",
       "\n",
       "   DAYS_EMPLOYED_DIFF_DAYS_BIRTH_SURESH  TREND_COEF_PAID_LATE_SURESH  \\\n",
       "0                                8824.0                     0.157343   \n",
       "1                               15577.0                          NaN   \n",
       "2                               18821.0                          NaN   \n",
       "3                               15966.0                     1.181081   \n",
       "4                               16894.0                    -0.504714   \n",
       "\n",
       "   AMT_INCOME_TOTAL_12_AMT_ANNUITY_SURESH  \\\n",
       "0                                 -7825.5   \n",
       "1                                -13198.5   \n",
       "2                                 -1125.0   \n",
       "3                                -18436.5   \n",
       "4                                -11740.5   \n",
       "\n",
       "   number_of_installments_paid_late_03  mean_number_of_days_paid_late_03  \\\n",
       "0                                  NaN                               NaN   \n",
       "1                                  NaN                               NaN   \n",
       "2                                  NaN                               NaN   \n",
       "3                                  NaN                               NaN   \n",
       "4                                  NaN                               NaN   \n",
       "\n",
       "   max_number_of_days_paid_late_03 mean_amont_paid_late_03  \n",
       "0                              NaN                     NaN  \n",
       "1                              NaN                     NaN  \n",
       "2                              NaN                     NaN  \n",
       "3                              NaN                     NaN  \n",
       "4                              NaN                     NaN  \n",
       "\n",
       "[5 rows x 362 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neptune.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniuqes = [f for f in neptune.columns if f not in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, neptune[uniuqes]], axis=1)\n",
    "\n",
    "#df = neptune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 884)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from keras.callbacks import Callback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import keras as ks\n",
    "from keras import backend as K\n",
    "import gc\n",
    "from contextlib import contextmanager\n",
    "import tensorflow as tf\n",
    "\n",
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting ANN. Train shape: (307511, 884), test shape: (48744, 884)\n",
      "(307511, 881)\n",
      "(246008, 1036) (61503, 1036) (48744, 1036)\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/5\n",
      "246008/246008 [==============================] - 36s 146us/step - loss: 0.2689 - binary_crossentropy: 0.2689 - val_loss: 0.2551 - val_binary_crossentropy: 0.2551\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.748999\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25515, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246008/246008 [==============================] - 35s 142us/step - loss: 0.2506 - binary_crossentropy: 0.2506 - val_loss: 0.2541 - val_binary_crossentropy: 0.2541\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.753844\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25515 to 0.25405, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246008/246008 [==============================] - 35s 142us/step - loss: 0.2485 - binary_crossentropy: 0.2485 - val_loss: 0.2520 - val_binary_crossentropy: 0.2520\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.758320\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.25405 to 0.25203, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246008/246008 [==============================] - 35s 142us/step - loss: 0.2476 - binary_crossentropy: 0.2476 - val_loss: 0.2514 - val_binary_crossentropy: 0.2514\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.759575\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.25203 to 0.25138, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246008/246008 [==============================] - 35s 142us/step - loss: 0.2469 - binary_crossentropy: 0.2469 - val_loss: 0.2510 - val_binary_crossentropy: 0.2510\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.760529\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.25138 to 0.25097, saving model to best_model.hdf5\n",
      "[{pass 1}] done in {202.339} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/5\n",
      "246008/246008 [==============================] - 33s 136us/step - loss: 0.2451 - binary_crossentropy: 0.2451 - val_loss: 0.2497 - val_binary_crossentropy: 0.2497\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.763523\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.25097 to 0.24971, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246008/246008 [==============================] - 33s 134us/step - loss: 0.2446 - binary_crossentropy: 0.2446 - val_loss: 0.2497 - val_binary_crossentropy: 0.2497\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.763828\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.24971\n",
      "Epoch 3/5\n",
      "246008/246008 [==============================] - 33s 134us/step - loss: 0.2441 - binary_crossentropy: 0.2441 - val_loss: 0.2495 - val_binary_crossentropy: 0.2495\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.764317\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24971 to 0.24950, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246008/246008 [==============================] - 33s 134us/step - loss: 0.2432 - binary_crossentropy: 0.2432 - val_loss: 0.2501 - val_binary_crossentropy: 0.2501\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.764699\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24950\n",
      "Epoch 5/5\n",
      "246008/246008 [==============================] - 33s 134us/step - loss: 0.2432 - binary_crossentropy: 0.2432 - val_loss: 0.2495 - val_binary_crossentropy: 0.2495\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.764669\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24950\n",
      "[{pass 2}] done in {190.373} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/5\n",
      "246008/246008 [==============================] - 33s 133us/step - loss: 0.2424 - binary_crossentropy: 0.2424 - val_loss: 0.2492 - val_binary_crossentropy: 0.2492\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.765908\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24950 to 0.24922, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246008/246008 [==============================] - 33s 132us/step - loss: 0.2422 - binary_crossentropy: 0.2422 - val_loss: 0.2488 - val_binary_crossentropy: 0.2488\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.766192\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24922 to 0.24884, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246008/246008 [==============================] - 33s 132us/step - loss: 0.2419 - binary_crossentropy: 0.2419 - val_loss: 0.2501 - val_binary_crossentropy: 0.2501\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.766722\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24884\n",
      "Epoch 4/5\n",
      "246008/246008 [==============================] - 33s 132us/step - loss: 0.2417 - binary_crossentropy: 0.2417 - val_loss: 0.2489 - val_binary_crossentropy: 0.2489\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.766758\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24884\n",
      "[{pass 3}] done in {150.693} s\n",
      "0.7667576655335868\n",
      "[{fit_predict}] done in {558.042} s\n",
      "(246009, 1035) (61502, 1035) (48744, 1035)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 36s 146us/step - loss: 0.2702 - binary_crossentropy: 0.2702 - val_loss: 0.2507 - val_binary_crossentropy: 0.2507\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.752651\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25068, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2516 - binary_crossentropy: 0.2516 - val_loss: 0.2486 - val_binary_crossentropy: 0.2486\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.757671\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25068 to 0.24857, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2497 - binary_crossentropy: 0.2497 - val_loss: 0.2479 - val_binary_crossentropy: 0.2479\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.759745\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24857 to 0.24789, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2484 - binary_crossentropy: 0.2484 - val_loss: 0.2473 - val_binary_crossentropy: 0.2473\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.761163\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24789 to 0.24730, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2478 - binary_crossentropy: 0.2478 - val_loss: 0.2471 - val_binary_crossentropy: 0.2471\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.760602\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24730 to 0.24711, saving model to best_model.hdf5\n",
      "[{pass 1}] done in {202.702} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 34s 137us/step - loss: 0.2466 - binary_crossentropy: 0.2466 - val_loss: 0.2466 - val_binary_crossentropy: 0.2466\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.764006\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24711 to 0.24659, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2456 - binary_crossentropy: 0.2456 - val_loss: 0.2458 - val_binary_crossentropy: 0.2458\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.764880\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24659 to 0.24584, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2451 - binary_crossentropy: 0.2451 - val_loss: 0.2457 - val_binary_crossentropy: 0.2457\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.765567\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24584 to 0.24565, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 33s 136us/step - loss: 0.2447 - binary_crossentropy: 0.2447 - val_loss: 0.2462 - val_binary_crossentropy: 0.2462\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.766018\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24565\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2441 - binary_crossentropy: 0.2441 - val_loss: 0.2462 - val_binary_crossentropy: 0.2462\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.764943\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24565\n",
      "[{pass 2}] done in {192.951} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2433 - binary_crossentropy: 0.2433 - val_loss: 0.2457 - val_binary_crossentropy: 0.2457\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.766705\n",
      "\n",
      "Epoch 00001: val_loss did not improve from 0.24565\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2429 - binary_crossentropy: 0.2429 - val_loss: 0.2455 - val_binary_crossentropy: 0.2455\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.766850\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24565 to 0.24554, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2428 - binary_crossentropy: 0.2428 - val_loss: 0.2452 - val_binary_crossentropy: 0.2452\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.767115\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24554 to 0.24516, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2422 - binary_crossentropy: 0.2422 - val_loss: 0.2457 - val_binary_crossentropy: 0.2457\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.767298\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24516\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2424 - binary_crossentropy: 0.2424 - val_loss: 0.2450 - val_binary_crossentropy: 0.2450\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.768429\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24516 to 0.24500, saving model to best_model.hdf5\n",
      "[{pass 3}] done in {190.011} s\n",
      "0.7684294853154781\n",
      "[{fit_predict}] done in {600.1} s\n",
      "(246009, 1036) (61502, 1036) (48744, 1036)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 36s 145us/step - loss: 0.2701 - binary_crossentropy: 0.2701 - val_loss: 0.2481 - val_binary_crossentropy: 0.2481\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.745545\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24814, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 35s 143us/step - loss: 0.2519 - binary_crossentropy: 0.2519 - val_loss: 0.2470 - val_binary_crossentropy: 0.2470\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.749324\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24814 to 0.24704, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 35s 143us/step - loss: 0.2501 - binary_crossentropy: 0.2501 - val_loss: 0.2467 - val_binary_crossentropy: 0.2467\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.752790\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24704 to 0.24675, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 35s 144us/step - loss: 0.2490 - binary_crossentropy: 0.2490 - val_loss: 0.2452 - val_binary_crossentropy: 0.2452\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.754567\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24675 to 0.24524, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 35s 144us/step - loss: 0.2483 - binary_crossentropy: 0.2483 - val_loss: 0.2452 - val_binary_crossentropy: 0.2452\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.755739\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24524 to 0.24519, saving model to best_model.hdf5\n",
      "[{pass 1}] done in {204.185} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2463 - binary_crossentropy: 0.2463 - val_loss: 0.2444 - val_binary_crossentropy: 0.2444\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.757350\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24519 to 0.24441, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2460 - binary_crossentropy: 0.2460 - val_loss: 0.2441 - val_binary_crossentropy: 0.2441\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.758399\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24441 to 0.24409, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2452 - binary_crossentropy: 0.2452 - val_loss: 0.2455 - val_binary_crossentropy: 0.2455\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.759286\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24409\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2452 - binary_crossentropy: 0.2452 - val_loss: 0.2438 - val_binary_crossentropy: 0.2438\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.759840\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24409 to 0.24380, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 33s 136us/step - loss: 0.2445 - binary_crossentropy: 0.2445 - val_loss: 0.2437 - val_binary_crossentropy: 0.2437\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.759631\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24380 to 0.24371, saving model to best_model.hdf5\n",
      "[{pass 2}] done in {192.298} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2438 - binary_crossentropy: 0.2438 - val_loss: 0.2432 - val_binary_crossentropy: 0.2432\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.760795\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24371 to 0.24325, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2434 - binary_crossentropy: 0.2434 - val_loss: 0.2440 - val_binary_crossentropy: 0.2440\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.761089\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.24325\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2429 - binary_crossentropy: 0.2429 - val_loss: 0.2440 - val_binary_crossentropy: 0.2440\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.760988\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24325\n",
      "[{pass 3}] done in {113.805} s\n",
      "0.7609875599409428\n",
      "[{fit_predict}] done in {524.424} s\n",
      "(246009, 1036) (61502, 1036) (48744, 1036)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 35s 144us/step - loss: 0.2697 - binary_crossentropy: 0.2697 - val_loss: 0.2502 - val_binary_crossentropy: 0.2502\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.749962\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25025, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2519 - binary_crossentropy: 0.2519 - val_loss: 0.2478 - val_binary_crossentropy: 0.2478\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.755836\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25025 to 0.24781, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2497 - binary_crossentropy: 0.2497 - val_loss: 0.2467 - val_binary_crossentropy: 0.2467\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.758992\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24781 to 0.24672, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2491 - binary_crossentropy: 0.2491 - val_loss: 0.2473 - val_binary_crossentropy: 0.2473\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.761428\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24672\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2481 - binary_crossentropy: 0.2481 - val_loss: 0.2454 - val_binary_crossentropy: 0.2454\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.763578\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24672 to 0.24542, saving model to best_model.hdf5\n",
      "[{pass 1}] done in {201.026} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 33s 136us/step - loss: 0.2459 - binary_crossentropy: 0.2459 - val_loss: 0.2458 - val_binary_crossentropy: 0.2458\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.764930\n",
      "\n",
      "Epoch 00001: val_loss did not improve from 0.24542\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2457 - binary_crossentropy: 0.2457 - val_loss: 0.2452 - val_binary_crossentropy: 0.2452\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.764973\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24542 to 0.24517, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2450 - binary_crossentropy: 0.2450 - val_loss: 0.2445 - val_binary_crossentropy: 0.2445\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.765679\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24517 to 0.24445, saving model to best_model.hdf5\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2446 - binary_crossentropy: 0.2446 - val_loss: 0.2443 - val_binary_crossentropy: 0.2443\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.766446\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24445 to 0.24433, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 33s 135us/step - loss: 0.2442 - binary_crossentropy: 0.2442 - val_loss: 0.2446 - val_binary_crossentropy: 0.2446\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.765351\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24433\n",
      "[{pass 2}] done in {191.829} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2434 - binary_crossentropy: 0.2434 - val_loss: 0.2442 - val_binary_crossentropy: 0.2442\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.766924\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24433 to 0.24416, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2429 - binary_crossentropy: 0.2429 - val_loss: 0.2441 - val_binary_crossentropy: 0.2441\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.766661\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24416 to 0.24410, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2428 - binary_crossentropy: 0.2428 - val_loss: 0.2443 - val_binary_crossentropy: 0.2443\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.767578\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24410\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2425 - binary_crossentropy: 0.2425 - val_loss: 0.2441 - val_binary_crossentropy: 0.2441\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.767567\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24410 to 0.24409, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 33s 134us/step - loss: 0.2425 - binary_crossentropy: 0.2425 - val_loss: 0.2440 - val_binary_crossentropy: 0.2440\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.767501\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24409 to 0.24396, saving model to best_model.hdf5\n",
      "[{pass 3}] done in {189.804} s\n",
      "0.7675014972455532\n",
      "[{fit_predict}] done in {596.807} s\n",
      "(246009, 1036) (61502, 1036) (48744, 1036)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 35s 143us/step - loss: 0.2709 - binary_crossentropy: 0.2709 - val_loss: 0.2469 - val_binary_crossentropy: 0.2469\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.753550\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24694, saving model to best_model.hdf5\n",
      "Epoch 2/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2523 - binary_crossentropy: 0.2523 - val_loss: 0.2454 - val_binary_crossentropy: 0.2454\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.758683\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24694 to 0.24543, saving model to best_model.hdf5\n",
      "Epoch 3/5\n",
      "246009/246009 [==============================] - 35s 143us/step - loss: 0.2508 - binary_crossentropy: 0.2508 - val_loss: 0.2456 - val_binary_crossentropy: 0.2456\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.759788\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24543\n",
      "Epoch 4/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2493 - binary_crossentropy: 0.2493 - val_loss: 0.2445 - val_binary_crossentropy: 0.2445\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.761386\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24543 to 0.24449, saving model to best_model.hdf5\n",
      "Epoch 5/5\n",
      "246009/246009 [==============================] - 35s 142us/step - loss: 0.2484 - binary_crossentropy: 0.2484 - val_loss: 0.2430 - val_binary_crossentropy: 0.2430\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.764036\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24449 to 0.24295, saving model to best_model.hdf5\n",
      "[{pass 1}] done in {201.23} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/5\n",
      "246009/246009 [==============================] - 33s 136us/step - loss: 0.2465 - binary_crossentropy: 0.2465 - val_loss: 0.2430 - val_binary_crossentropy: 0.2430\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.764045\n",
      "\n",
      "Epoch 00001: val_loss did not improve from 0.24295\n",
      "Epoch 2/5\n",
      " 36864/246009 [===>..........................] - ETA: 26s - loss: 0.2439 - binary_crossentropy: 0.2439"
     ]
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "gc.collect()\n",
    "train_df = df.iloc[0:n_train]\n",
    "test_df = df.iloc[n_train:]\n",
    "\n",
    "print(\"Starting ANN. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train['TARGET'])):\n",
    "        \n",
    "        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.OneHotEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            x_test = enc.transform(test_df[feats]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            gc.collect()\n",
    "            scaler = preprocessing.RobustScaler(quantile_range=(5.0, 95.0), with_scaling=True, with_centering=True)\n",
    "            scaler.fit(x_train)\n",
    "            x_train = scaler.transform(x_train)\n",
    "            x_valid = scaler.transform(x_valid)\n",
    "            x_test = scaler.transform(x_test)\n",
    "            \n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "        \n",
    "        file_path = \"best_model.hdf5\"\n",
    "        check_point = ModelCheckpoint(file_path, monitor=\"val_loss\", verbose=1,\n",
    "                              save_best_only=True, mode=\"min\")\n",
    "        ra_val = RocAucEvaluation(validation_data=(x_valid, train['TARGET'].iloc[valid_idx].values), interval=1)\n",
    "        early_stop = EarlyStopping(monitor=\"val_loss\", mode = \"min\", patience=2)\n",
    "        gc.collect()\n",
    "        \n",
    "        config = tf.ConfigProto(\n",
    "        intra_op_parallelism_threads=6, use_per_session_threads=6, inter_op_parallelism_threads=6)\n",
    "        with tf.Session(graph=tf.Graph(), config=config) as sess, timer('fit_predict'):\n",
    "                    ks.backend.set_session(sess)\n",
    "                    model_in = ks.Input(shape=(x_train.shape[1],), dtype='float32', sparse=False)\n",
    "                    out = ks.layers.Dense(2 ** 10,  activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(model_in)\n",
    "                    out = ks.layers.Dropout(0.5)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 9, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.3)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 8, activation='relu', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.2)(out)\n",
    "                    out = ks.layers.Dense(1, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    model = ks.models.Model(model_in, out)\n",
    "                    model.compile(loss='binary_crossentropy',\n",
    "                                  optimizer=ks.optimizers.Adam(lr=1e-3), metrics=['binary_crossentropy'])\n",
    "                    batch_size = 2 ** 10\n",
    "                    for i in range(3):\n",
    "                        with timer('pass ' +  str(i + 1)):\n",
    "                            model.fit(x=x_train, y=train['TARGET'].iloc[train_idx].values, batch_size=batch_size+(batch_size*(2*i)), epochs=5, \n",
    "                                validation_data=(x_valid, train['TARGET'].iloc[valid_idx].values), callbacks=[ra_val, check_point, early_stop], shuffle=True)\n",
    "                    y_pred = model.predict(x_valid).reshape(-1, 1)\n",
    "                    print(roc_auc_score(y_true=train['TARGET'].iloc[valid_idx].values, y_score=y_pred))\n",
    "                    gc.collect()   \n",
    "                    oof_preds[valid_idx] = model.predict(x_valid)[:, 0]\n",
    "                    sub_preds += model.predict(x_test)[:, 0] / folds.n_splits\n",
    "                    gc.collect()\n",
    "\n",
    "print('Full AUC score %.6f' % roc_auc_score(train['TARGET'], oof_preds))\n",
    "             \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Full AUC score %.6f' % roc_auc_score(train['TARGET'], oof_preds))\n",
    "        \n",
    "sub_df = test[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "val_df = train[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "            \n",
    "        \n",
    "        \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
